from pymongo import MongoClient
from multiprocessing import Pool
from multiprocessing import Process
from selenium import webdriver
from bs4 import BeautifulSoup

import requests
import pprint
import random
import time
import json
import re

client = MongoClient('127.0.0.1', 27017, connect = False)
douban_book_db = client['douban_book_db']
douban_book_coll = douban_book_db['douban_book_coll']


class DoubanBook:
    def __init__(self, tags):
        self.driver = webdriver.Chrome(executable_path = '/Users/xuchaosheng/Workspace/qushu-recruit-data/chromedriver')
        self.driver.get('https://accounts.douban.com/login')
        time.sleep(30)

        for tag in tags:
            self.book_list(tag, 0)

    def book_list(self, tag, start):
        url = 'https://book.douban.com/tag/' + tag + '?start=' + str(start) + '&type=T'
        self.driver.get(url)
        soup = BeautifulSoup(self.driver.page_source)

        if not len(soup.select('.subject-item')):
            return

        for item in soup.select('.subject-item'):
            book_id = re.sub("\D", "", item.find('a').get('href'))
            self.book_info(book_id)

        self.book_list(tag, start + 20)

    def book_info(self, id):
        if douban_book_coll.find_one({'book_id': id}):
            return

        self.driver.get('https://book.douban.com/subject/' + id + '/')
        soup = BeautifulSoup(self.driver.page_source)

        title = soup.find('h1').get_text().strip()

        if not title:
            print('请检查页面')
            return

        try:
            author = soup.find(id = 'content').find(id = 'info').find(text = re.compile('作者')).parent.parent.find('a').get_text().strip()
        except:
            author = ''

        try:
            basic_info = soup.find(id = 'content').find(id = 'info').find(text = re.compile('出版社:')).parent.parent.get_text().strip()
        except:
            basic_info = ''

        try:
            rating_num = soup.find(class_ = 'rating_num').text.strip()
        except:
            rating_num = ''

        try:
            book_intro = soup.find('span', text = re.compile('内容简介')).parent.find_next_sibling().find(class_ = 'intro').get_text().strip()
        except:
            book_intro = ''

        try:
            author_intro = soup.find('span', text = re.compile('作者简介')).parent.find_next_sibling().find(class_ = 'intro').get_text().strip()
        except:
            author_intro = ''
        try:
            book_menu = soup.find('span', text = re.compile('目录')).parent.find_next_sibling().find_next_sibling().get_text().strip()
        except:
            book_menu = ''

        try:
            labels = [i.text for i in soup.find('span', text = re.compile('豆瓣成员常用的标签')).parent.find_next_sibling().select('.tag')]
        except:
            labels = ''

        try:
            book_pic = soup.find(id = 'mainpic').find('img').get('src')
        except:
            book_pic = ''

        item = {
            'book_id': id,
            'title': title,
            'author': author,
            'basic_info': basic_info,
            'rating_num': rating_num,
            'book_intro': book_intro,
            'author_intro': author_intro,
            'book_menu': book_menu,
            'labels': labels,
            'book_pic': book_pic,
            'comments': self.book_comments(id)
        }

        pprint.pprint(item)

        if not douban_book_coll.find_one({'book_id': id}):
            douban_book_coll.insert_one(item)
        else:
            douban_book_coll.update_one({'book_id': id}, {'$set': {
                'comments': item['comments']
            }})

    def book_comments(self, id):
        comments = []

        for i in range(1, 30):
            flag = True

            for t in ['hot']:
                self.driver.get('https://book.douban.com/subject/' + id + '/comments/' + t + '?p=' + str(i))
                time.sleep(1)

                if '还没人写过短评呢' in self.driver.page_source:
                    return comments

                soup = BeautifulSoup(self.driver.page_source)

                if not len(soup.select('.comment-item')):
                    flag = False
                    break

                for item in soup.select('.comment-item'):
                    com = {
                        'text': item.find(class_ = 'comment-content').get_text().strip() if item.find(class_ = 'comment-content') else '',
                        'time': item.find(class_ = 'comment-info').select('span')[-1].text if item.find(class_ = 'comment-info') else '',
                        'stars': int(re.sub("\D", "", item.find(class_ = 'user-stars').get('class')[1])) / 10 if item.find(class_ = 'user-stars') else '',
                    }

                    comments.append(com)

                    pprint.pprint(com)
                    print('=' * 100)

            if not flag:
                break

        return comments


def start(tags):
    DoubanBook(tags)


tags = [
    '小说',
    '外国文学',
    '文学',
    '随笔',
    '中国文学',
    '经典',
    '日本文学',
    '散文',
    '村上春树',
    '诗歌',
    '童话',
    '儿童文学',
    '古典文学',
    '名著',
    '王小波',
    '杂文',
    '余华',
    '张爱玲',
    '当代文学',
    '钱钟书',
    '外国名著',
    '鲁迅',
    '诗词',
    '茨威格',
    '米兰·昆德拉',
    '杜拉斯',
    '港台',
    '漫画',
    '推理',
    '绘本',
    '青春',
    '东野圭吾',
    '科幻',
    '言情',
    '悬疑',
    '奇幻',
    '武侠',
    '日本漫画',
    '推理小说',
    '韩寒',
    '耽美',
    '亦舒',
    '网络小说',
    '三毛',
    '安妮宝贝',
    '阿加莎·克里斯蒂',
    '穿越',
    '郭敬明',
    '金庸',
    '科幻小说',
    '轻小说',
    '青春文学',
    '魔幻',
    '几米',
    '幾米',
    '张小娴',
    'J.K.罗琳',
    '古龙',
    '高木直子',
    '沧月',
    '校园',
    '落落',
    '张悦然',
    '历史',
    '心理学',
    '哲学',
    '传记',
    '文化',
    '社会学',
    '艺术',
    '社会',
    '设计',
    '政治',
    '建筑',
    '宗教',
    '电影',
    '政治学',
    '数学',
    '中国历史',
    '回忆录',
    '思想',
    '国学',
    '人物传记',
    '人文',
    '音乐',
    '艺术史',
    '绘画',
    '戏剧',
    '西方哲学',
    '二战',
    '军事',
    '佛教',
    '近代史',
    '考古',
    '自由主义',
    '美术',
    '爱情',
    '旅行',
    '成长',
    '生活',
    '心理',
    '励志',
    '女性',
    '摄影',
    '教育',
    '职场',
    '美食',
    '游记',
    '灵修',
    '健康',
    '情感',
    '两性',
    '人际关系',
    '手工',
    '养生',
    '家居',
    '自助游',
    '经济学',
    '管理',
    '经济',
    '商业',
    '金融',
    '投资',
    '营销',
    '理财',
    '创业',
    '广告',
    '股票',
    '企业史',
    '策划',
    '科普',
    '互联网',
    '编程',
    '科学',
    '交互设计',
    '用户体验',
    '算法',
    '科技',
    'web',
    'UE',
    '交互',
    '通信',
    'UCD',
    '神经网络',
    '程序'
]

if __name__ == '__main__':
    # for i in douban_book_coll.aggregate([{'$sample': {'size': 10}}]):
    #     start(i['book_id'])
    random.shuffle(tags)
    start(tags)
